{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "Iter0, Testing Accuracy: 0.9375, Training Accuracy: 0.93545455\n",
      "Iter1, Testing Accuracy: 0.945, Training Accuracy: 0.94641817\n",
      "Iter2, Testing Accuracy: 0.9512, Training Accuracy: 0.95467275\n",
      "Iter3, Testing Accuracy: 0.9546, Training Accuracy: 0.96009094\n",
      "Iter4, Testing Accuracy: 0.9637, Training Accuracy: 0.9691273\n",
      "Iter5, Testing Accuracy: 0.9661, Training Accuracy: 0.9715091\n",
      "Iter6, Testing Accuracy: 0.9659, Training Accuracy: 0.97114545\n",
      "Iter7, Testing Accuracy: 0.9688, Training Accuracy: 0.9737818\n",
      "Iter8, Testing Accuracy: 0.9689, Training Accuracy: 0.9757636\n",
      "Iter9, Testing Accuracy: 0.9707, Training Accuracy: 0.9798909\n",
      "Iter10, Testing Accuracy: 0.9737, Training Accuracy: 0.9816909\n",
      "Iter11, Testing Accuracy: 0.9724, Training Accuracy: 0.98154545\n",
      "Iter12, Testing Accuracy: 0.974, Training Accuracy: 0.98343635\n",
      "Iter13, Testing Accuracy: 0.9756, Training Accuracy: 0.9842727\n",
      "Iter14, Testing Accuracy: 0.9748, Training Accuracy: 0.9845273\n",
      "Iter15, Testing Accuracy: 0.9741, Training Accuracy: 0.98374546\n",
      "Iter16, Testing Accuracy: 0.9768, Training Accuracy: 0.9862\n",
      "Iter17, Testing Accuracy: 0.9758, Training Accuracy: 0.9863273\n",
      "Iter18, Testing Accuracy: 0.9755, Training Accuracy: 0.98652726\n",
      "Iter19, Testing Accuracy: 0.9754, Training Accuracy: 0.9875091\n",
      "Iter20, Testing Accuracy: 0.9792, Training Accuracy: 0.9884909\n",
      "Iter21, Testing Accuracy: 0.977, Training Accuracy: 0.9881818\n",
      "Iter22, Testing Accuracy: 0.9776, Training Accuracy: 0.98892725\n",
      "Iter23, Testing Accuracy: 0.9777, Training Accuracy: 0.9895818\n",
      "Iter24, Testing Accuracy: 0.9781, Training Accuracy: 0.9894\n",
      "Iter25, Testing Accuracy: 0.9769, Training Accuracy: 0.9900727\n",
      "Iter26, Testing Accuracy: 0.9776, Training Accuracy: 0.99023634\n",
      "Iter27, Testing Accuracy: 0.9787, Training Accuracy: 0.9909091\n",
      "Iter28, Testing Accuracy: 0.9791, Training Accuracy: 0.9909273\n",
      "Iter29, Testing Accuracy: 0.9782, Training Accuracy: 0.99136364\n",
      "Iter30, Testing Accuracy: 0.9801, Training Accuracy: 0.99161816\n",
      "Iter31, Testing Accuracy: 0.979, Training Accuracy: 0.9918727\n",
      "Iter32, Testing Accuracy: 0.9786, Training Accuracy: 0.9921273\n",
      "Iter33, Testing Accuracy: 0.9795, Training Accuracy: 0.99218184\n",
      "Iter34, Testing Accuracy: 0.9799, Training Accuracy: 0.9931818\n",
      "Iter35, Testing Accuracy: 0.9797, Training Accuracy: 0.9920545\n",
      "Iter36, Testing Accuracy: 0.9779, Training Accuracy: 0.99274546\n",
      "Iter37, Testing Accuracy: 0.9804, Training Accuracy: 0.9932\n",
      "Iter38, Testing Accuracy: 0.9797, Training Accuracy: 0.9936182\n",
      "Iter39, Testing Accuracy: 0.9811, Training Accuracy: 0.9936182\n",
      "Iter40, Testing Accuracy: 0.9808, Training Accuracy: 0.994\n",
      "Iter41, Testing Accuracy: 0.9797, Training Accuracy: 0.9938727\n",
      "Iter42, Testing Accuracy: 0.9817, Training Accuracy: 0.99445456\n",
      "Iter43, Testing Accuracy: 0.9816, Training Accuracy: 0.99472725\n",
      "Iter44, Testing Accuracy: 0.9803, Training Accuracy: 0.9940364\n",
      "Iter45, Testing Accuracy: 0.9802, Training Accuracy: 0.9947091\n",
      "Iter46, Testing Accuracy: 0.9804, Training Accuracy: 0.99456364\n",
      "Iter47, Testing Accuracy: 0.9823, Training Accuracy: 0.9950182\n",
      "Iter48, Testing Accuracy: 0.9808, Training Accuracy: 0.9944909\n",
      "Iter49, Testing Accuracy: 0.9812, Training Accuracy: 0.9950182\n",
      "Iter50, Testing Accuracy: 0.9814, Training Accuracy: 0.9950727\n",
      "Iter51, Testing Accuracy: 0.9808, Training Accuracy: 0.99538183\n",
      "Iter52, Testing Accuracy: 0.9814, Training Accuracy: 0.9954364\n",
      "Iter53, Testing Accuracy: 0.9814, Training Accuracy: 0.99527276\n",
      "Iter54, Testing Accuracy: 0.9811, Training Accuracy: 0.99545455\n",
      "Iter55, Testing Accuracy: 0.9801, Training Accuracy: 0.99503636\n",
      "Iter56, Testing Accuracy: 0.9812, Training Accuracy: 0.9954727\n",
      "Iter57, Testing Accuracy: 0.9816, Training Accuracy: 0.9958364\n",
      "Iter58, Testing Accuracy: 0.9828, Training Accuracy: 0.99596363\n",
      "Iter59, Testing Accuracy: 0.9822, Training Accuracy: 0.99605453\n",
      "Iter60, Testing Accuracy: 0.9817, Training Accuracy: 0.9958909\n",
      "Iter61, Testing Accuracy: 0.9812, Training Accuracy: 0.9960182\n",
      "Iter62, Testing Accuracy: 0.9824, Training Accuracy: 0.9960727\n",
      "Iter63, Testing Accuracy: 0.9811, Training Accuracy: 0.9963091\n",
      "Iter64, Testing Accuracy: 0.9813, Training Accuracy: 0.9962\n",
      "Iter65, Testing Accuracy: 0.9827, Training Accuracy: 0.99618185\n",
      "Iter66, Testing Accuracy: 0.9826, Training Accuracy: 0.99627274\n",
      "Iter67, Testing Accuracy: 0.9828, Training Accuracy: 0.9962909\n",
      "Iter68, Testing Accuracy: 0.9818, Training Accuracy: 0.9963273\n",
      "Iter69, Testing Accuracy: 0.9825, Training Accuracy: 0.9963091\n",
      "Iter70, Testing Accuracy: 0.9815, Training Accuracy: 0.9963273\n",
      "Iter71, Testing Accuracy: 0.9813, Training Accuracy: 0.99636364\n",
      "Iter72, Testing Accuracy: 0.9831, Training Accuracy: 0.9964909\n",
      "Iter73, Testing Accuracy: 0.9836, Training Accuracy: 0.99665457\n",
      "Iter74, Testing Accuracy: 0.9822, Training Accuracy: 0.99654543\n",
      "Iter75, Testing Accuracy: 0.9812, Training Accuracy: 0.99652725\n",
      "Iter76, Testing Accuracy: 0.9832, Training Accuracy: 0.9966909\n",
      "Iter77, Testing Accuracy: 0.9835, Training Accuracy: 0.9968\n",
      "Iter78, Testing Accuracy: 0.9815, Training Accuracy: 0.99674547\n",
      "Iter79, Testing Accuracy: 0.983, Training Accuracy: 0.9968\n",
      "Iter80, Testing Accuracy: 0.9812, Training Accuracy: 0.9967273\n",
      "Iter81, Testing Accuracy: 0.983, Training Accuracy: 0.9969636\n",
      "Iter82, Testing Accuracy: 0.9816, Training Accuracy: 0.99676365\n",
      "Iter83, Testing Accuracy: 0.9825, Training Accuracy: 0.9969818\n",
      "Iter84, Testing Accuracy: 0.983, Training Accuracy: 0.99707276\n",
      "Iter85, Testing Accuracy: 0.9828, Training Accuracy: 0.99707276\n",
      "Iter86, Testing Accuracy: 0.9825, Training Accuracy: 0.99701816\n",
      "Iter87, Testing Accuracy: 0.982, Training Accuracy: 0.9969818\n",
      "Iter88, Testing Accuracy: 0.9814, Training Accuracy: 0.99701816\n",
      "Iter89, Testing Accuracy: 0.9824, Training Accuracy: 0.99709094\n",
      "Iter90, Testing Accuracy: 0.9825, Training Accuracy: 0.9971091\n",
      "Iter91, Testing Accuracy: 0.9827, Training Accuracy: 0.9972\n",
      "Iter92, Testing Accuracy: 0.9822, Training Accuracy: 0.99718183\n",
      "Iter93, Testing Accuracy: 0.9823, Training Accuracy: 0.9972182\n",
      "Iter94, Testing Accuracy: 0.9831, Training Accuracy: 0.9972364\n",
      "Iter95, Testing Accuracy: 0.9819, Training Accuracy: 0.9972\n",
      "Iter96, Testing Accuracy: 0.9834, Training Accuracy: 0.99718183\n",
      "Iter97, Testing Accuracy: 0.9827, Training Accuracy: 0.9973091\n",
      "Iter98, Testing Accuracy: 0.9822, Training Accuracy: 0.99732727\n",
      "Iter99, Testing Accuracy: 0.9824, Training Accuracy: 0.99734545\n",
      "Iter100, Testing Accuracy: 0.9821, Training Accuracy: 0.9973818\n"
     ]
    }
   ],
   "source": [
    "#load dataset\n",
    "mnist = input_data.read_data_sets(\"MNIST_data\",one_hot=True)\n",
    "\n",
    "#define batch size\n",
    "batch_size = 100\n",
    "#calculate number of batches\n",
    "n_batch = mnist.train.num_examples // batch_size\n",
    "\n",
    "#define placeholders\n",
    "x = tf.placeholder(tf.float32, [None,784])\n",
    "y = tf.placeholder(tf.float32, [None,10])\n",
    "keep_prob=tf.placeholder(tf.float32)\n",
    "\n",
    "#create simple NeuroNet\n",
    "W1 = tf.Variable(tf.truncated_normal([784,850],stddev=0.1))\n",
    "b1 = tf.Variable(tf.zeros([850])+0.1)\n",
    "L1 = tf.nn.tanh(tf.matmul(x,W1)+b1)\n",
    "L1_drop = tf.nn.dropout(L1,keep_prob)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([850,650],stddev=0.1))\n",
    "b2 = tf.Variable(tf.zeros([650])+0.1)\n",
    "L2 = tf.nn.tanh(tf.matmul(L1_drop,W2)+b2)\n",
    "L2_drop = tf.nn.dropout(L2,keep_prob)\n",
    "\n",
    "W3 = tf.Variable(tf.truncated_normal([650,10],stddev=0.1))\n",
    "b3 = tf.Variable(tf.zeros([10])+0.1)\n",
    "prediction = tf.nn.softmax(tf.matmul(L2_drop,W3)+b3)\n",
    "\n",
    "#cost function\n",
    "# loss = tf.reduce_mean(tf.square(y-prediction))\n",
    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=prediction))\n",
    "# train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "train_step = tf.train.AdagradOptimizer(0.5).minimize(loss)\n",
    "\n",
    "#initialize variables\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "#find accuracy of trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y,1),tf.argmax(prediction,1))   #convert a list of booleans into a single boolean value\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for epoch in range(101):\n",
    "        for batch in range(n_batch):\n",
    "            batch_xs,batch_ys = mnist.train.next_batch(batch_size)\n",
    "            sess.run(train_step,feed_dict={x:batch_xs,y:batch_ys,keep_prob:0.8})\n",
    "        \n",
    "        test_acc = sess.run(accuracy,feed_dict={x:mnist.test.images,y:mnist.test.labels,keep_prob:1.0})\n",
    "        train_acc = sess.run(accuracy,feed_dict={x:mnist.train.images,y:mnist.train.labels,keep_prob:1.0})\n",
    "\n",
    "        print(\"Iter\" + str(epoch) + \", Testing Accuracy: \" + str(test_acc) + \", Training Accuracy: \" +str(train_acc))\n",
    "        \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
